# Updated by Wei on 2014/02/13 afternoon at school
import os
from struct import *
import sys

# This script is used for reading/writing different kinds of index for aux use.
print "read/write binary file"

def simpleLook():
    print "simpleLook"    
    fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/LEAVE_allPostingsBeingPopped20140312Night_weight_10_WHOLE"
    inputFileHandler = open(fileName,"rb")
    tempCounter = 0
    for i in range(0,1024):
        s1 = inputFileHandler.read(4 + 4 + 4 + 4)
        (termID,docID,finalProbability,impactScore) = unpack( "2I2f", s1)
        postingKey = str(termID) + "_" + str(docID)
        print termID,docID,finalProbability,impactScore

        #if termID == 24657868 and docID == 2093428:
        #    print termID,docID,finalProbability,impactScore
        #    print "len(postingDict):",len(postingDict)
        #    exit(1)
        
        if termID == 24657868 and docID == 15826653:
            print termID,docID,finalProbability,impactScore
            
            exit(1)
        
        tempCounter += 1
        
        if tempCounter % 1000000 == 0:
            print "tempCounter:",tempCounter

    inputFileHandler.close()
    exit(1)
        

    fileName1 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/LEAVE_allPostingsBeingPopped20140310Afternoon_weight_0_WHOLE"
    fileName2 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/LEAVE_allPostingsBeingPopped20140312Night_weight_10_WHOLE"
    fileName3 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/LEAVE_allPostingsBeingPopped20140314Morning_weight_20_WHOLE"    
    inputFileHandler1 = open(fileName1,"rb")
    inputFileHandler2 = open(fileName2,"rb")
    inputFileHandler3 = open(fileName3,"rb")
    for i in range(0,100000):
        s1 = inputFileHandler1.read(4 + 4 + 4 + 4)
        s2 = inputFileHandler2.read(4 + 4 + 4 + 4)
        s3 = inputFileHandler3.read(4 + 4 + 4 + 4)
        (termID1,docID1,probablity1,impactScore1) = unpack( "2I2f", s1)
        (termID2,docID2,probablity2,impactScore2) = unpack( "2I2f", s2)
        (termID3,docID3,probablity3,impactScore3) = unpack( "2I2f", s3)
        print termID1,docID1,probablity1,impactScore1
        print termID2,docID2,probablity2,impactScore2
        print termID3,docID3,probablity3,impactScore3
        print
    exit(1)
    
    # fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/LEAVE_allPostingsBeingPopped20140310Afternoon_weight_0_WHOLE"
    fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/LEAVE_allPostingsBeingPopped20140312Night_weight_10_WHOLE"
    # fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/LEAVE_allPostingsBeingPopped20140314Morning_weight_20_WHOLE"
    inputFileHandler = open(fileName,"rb")
    for i in range(0,645194801):
        s1 = inputFileHandler.read(4 + 4 + 4 + 4)
        (termID,docID,probablity,impactScore) = unpack( "2I2f", s1)
        # print termID,docID,probablity,impactScore
        if termID == 24657868 and docID == 15826653:
            print termID,docID,probablity,impactScore
            exit(1)
    print "end"
    exit(1)

    fileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140320Afternoon_weight_50_WHOLE"
    inputFileHandler = open(fileName,"rb")
    for i in range(0,256):
        s1 = inputFileHandler.read(4 + 4 + 4 + 4)
        (termID,docID,finalProbability,impactScore) = unpack( "2I2f", s1)
        print termID,docID,finalProbability,impactScore
    inputFileHandler.close()
    exit(1)  
    
    
    postingDict = {}
    # fileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140318Afternoon_weight_50_WHOLE"
    # fileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140318Afternoon_weight_50_Part3"
    fileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140319Morning_weight_1_Part3"
    inputFileHandler = open(fileName,"rb")
    while True:
        s1 = inputFileHandler.read(4 + 4 + 4 + 4)
        (termID,docID,finalProbability,impactScore) = unpack( "2I2f", s1)
        postingKey = str(termID) + "_" + str(docID)
        # print termID,docID,finalProbability,impactScore

        if termID == 24657868 and docID == 2093428:
            print termID,docID,finalProbability,impactScore
            print "len(postingDict):",len(postingDict)
            exit(1)
        
        if termID == 24657868 and docID == 15826653:
            print termID,docID,finalProbability,impactScore
            print "len(postingDict):",len(postingDict)
            exit(1)
        
        if postingKey not in postingDict:
            postingDict[postingKey] = 1
        else:
            print "fuc*",postingKey
            exit(1)
        if len(postingDict) % 1000000 == 0:
            print "len(postingDict):",len(postingDict)
    inputFileHandler.close()
    exit(1)
    
    # fileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140318Afternoon_weight_50_WHOLE"
    # fileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140318Afternoon_weight_50_Part0_1"
    fileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140318Afternoon_weight_50_Part2"
    inputFileHandler = open(fileName,"rb")
    for i in range(0,1024):
        s1 = inputFileHandler.read(4 + 4 + 4 + 4)
        (termID,docID,finalProbability,impactScore) = unpack( "2i2f", s1)
        print termID,docID,finalProbability,impactScore
    inputFileHandler.close()
    exit(1)

    # fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/scripts/src/cScripts/resultfile0"
    # fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/scripts/src/cScripts/LEAVE_allPostingsBeingPopped20140304Night_Part1_DEBUG"
    # fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/scripts/src/cScripts/LEAVE_allPostingsBeingPopped20140304Night_Part2_DEBUG"
    # fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/scripts/src/cScripts/LEAVE_allPostingsBeingPopped20140304Night_Part3_DEBUG"
    # fileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/LEAVE_allPostingsBeingPopped20140304Night_Part1"
    fileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/LEAVE_selectedDocumentPostingValuesInfo20140306Afternoon_PART4_DOC.binary"
    inputFileHandler = open(fileName,"rb")
    '''
    numOfPostingsNeededToRead = 100
    for i in range(0,numOfPostingsNeededToRead):
        s = inputFileHandler.read(4 + 4 + 4)
        (termID,docID,finalProbablity) = unpack( "2I1f", s)
        print termID,docID,finalProbablity
    '''
    # inputFileHandler = open("/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/LEAVE_selectedDocumentPostingValuesInfo20140306Afternoon_PART2_DOC.binary","rb")
    s1 = inputFileHandler.read(4 + 4)
    (docID,numOfPostings) = unpack( "2I", s1)
    print "docID:",docID
    print "numOfPostings:",numOfPostings
    for i in range(0,338):
        s1 = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
        (termID,currentStaticProbablity,currentDynamicProbability,currentFinalProbability,partialBM25) = unpack( "1I4f", s1)
        print i,termID,currentStaticProbablity,currentDynamicProbability,currentFinalProbability,partialBM25
    
    s1 = inputFileHandler.read(4 + 4)
    (docID,numOfPostings) = unpack( "2I", s1)
    print "docID:",docID
    print "numOfPostings:",numOfPostings
    for i in range(0,97):
        s1 = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
        (termID,currentStaticProbablity,currentDynamicProbability,currentFinalProbability,partialBM25) = unpack( "1I4f", s1)
        print i,termID,currentStaticProbablity,currentDynamicProbability,currentFinalProbability,partialBM25
    
    exit(1)
    '''
    inputFileHandler.seek(333316040)
    s1 = inputFileHandler.read(4)
    (docID,) = unpack( "1I", s1)
    print "docID:",docID    
    '''

def read1():
    print "read1()"
    inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/scripts/src/cScripts/LEAVE_allPostingsBeingPopped20140310Afternoon_Part4_DEBUG"
    inputFileHandler = open(inputFileName,"r")
    for i in range(0,100):
        byteString = inputFileHandler.read(4 + 4 + 4 + 4)
        (termID,docID,finalProbablity,impactScore) = unpack( "2I2f", byteString)
        print termID,docID,finalProbablity,impactScore
    inputFileHandler.close()
    exit(1)

def read2():
    print "read2()"
    inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/allPostingsBeingPopped.binary"
    inputFileHandler = open(inputFileName,"rb")
    
    for j in range(0,63):
        string = inputFileHandler.read(4 + 4 + 4)
        (v1,v2,v3) = unpack( "2i1f", string)
        print j,v3,v1,v2
    
    inputFileHandler.close()

def read3():
    print "read3()"
    inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/allPostingsBeingPopped20140304Night_Part1"
    inputFileHandler = open(inputFileName,"r")
    
    statinfo = os.stat(inputFileName)
    fileSize = statinfo.st_size
    print "file size:",fileSize

    numOfBytesRead = 0
    
    while numOfBytesRead <= fileSize:
        # print "numOfBytesRead:",numOfBytesRead
        byteString = inputFileHandler.read(4 + 4 + 4)
        (termID,docID,probability) = unpack( "2I1f", byteString)
        print termID,docID,probability
        
        numOfBytesRead += 4 + 4 + 4
        
    inputFileHandler.close()

def write():
    print "write"
    outputFileHandler = open("/home/weijiang/workspace/polyIRToolkit/polyIRIndexer/testBi.ext","wb")
    outputFileHandler.write(pack("66f", 1.40129846432e-45, 15.9015455246, 15.9015455246,4.82374,5.14076,8.56119,3.10649,1.57827,5.12863,9.19079,6.88509,13.779,5.63311,5.11737,13.66,11.8439,16.0316,9.33557,15.2096,15.9656,5.40692,9.77615,15.2294,15.8159,15.4182,8.6593,11.7666,16.0253,0.511922,0.61234,8.2088,2.17585,6.71241,2.91754,5.81428,12.1112,11.6405,2.90003,7.83524,7.62165,2.90662,2.39162,15.1743,10.1396,11.7676,3.89207,11.2871,15.3711,4.10446,9.10909,0.825253,11.6249,1.2435,4.63435,1.9132,1.47068,15.1274,15.4972,11.2854,13.2075,1.58034,4.35082,3.31788,1.63549,0.755889,2.72143))
    outputFileHandler.close()

# Updated by Wei on 2014/03/07 afternoon
def readAndWrite():
    # outputFileName
    outputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/scripts/src/cScripts/LEAVE_allPostingsBeingPopped20140310Afternoon_Part4_DEBUG"
    outputFileHandler = open(outputFileName,"wb")
    
    # inputFileName
    inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/scripts/src/cScripts/LEAVE_allPostingsBeingPopped20140310Afternoon_Part4"
    inputFileHandler = open(inputFileName,"rb")
    
    numOfPostingsNeededToRead = 100
    for i in range(0,numOfPostingsNeededToRead):
        byteString = inputFileHandler.read(4 + 4 + 4 + 4)
        outputFileHandler.write(byteString)
    
    '''
    while currentLine:
        currentLineElements = currentLine.strip().split(" ") 
        # print "currentLineElements: ",currentLineElements
        docID = int(currentLineElements[0])
        numOfPostings = int(currentLineElements[1])
        
        # print "numOfPostings:",numOfPostings
        # print "len(currentLineElements):",len(currentLineElements)
        # exit(1)
        
        if (numOfPostings * 4 + 4) == len(currentLineElements):
            numOfBytes += 4 + 4 + numOfPostings * 3 * 4
            
            # print "complete",str(lineNumber),str(docID),str(numOfPostings)
            outputFileHandler.write(pack("2I",docID,numOfPostings))
            
            baseIndex = 2
            for index in range(0,numOfPostings):
                termID = int(currentLineElements[baseIndex])
                staticProbablity = float(currentLineElements[baseIndex+1])
                dynamicProbablity = float(currentLineElements[baseIndex+2])
                
                # print index,termID,staticProbablity,dynamicProbablity
                outputFileHandler.write(pack("Iff",termID,staticProbablity,dynamicProbablity))
                baseIndex += 4
        else:
            print "incomplete",str(lineNumber),str(docID),str(numOfPostings)
        currentLine = inputFileHandler.readline()
        lineNumber += 1
        
        # for debug
        # if lineNumber % 1000 == 0:
        #    break
        
        if lineNumber % 1000 == 0:
            print "docs processed",str(lineNumber)
    '''
    
    inputFileHandler.close()
    outputFileHandler.close()
    print "Overall:"
    print "inputFileName: ",inputFileName
    print "outputFileName: ",outputFileName
    
# Updated by Wei on 2014/03/07 morning
def readAndVerify():
    # option1:
    inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/selectedDocumentPostingValuesInfo20140306Afternoon_PART3_DOC.binary"
    # option2:
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/selectedDocumentPostingValuesInfo20140306Afternoon_PART_OF_DOC.binary"

    inputFileHandler = open(inputFileName,"rb")
    
    statinfo = os.stat(inputFileName)
    fileSize = statinfo.st_size
    print "file size:",fileSize
    
    num_of_bytes_read = 0
    
    while num_of_bytes_read < fileSize:
        byteString = inputFileHandler.read(4 + 4)
        (docID,numOfPostings) = unpack( "2I", byteString)
        num_of_bytes_read += 8
        
        if docID == 18500001 or docID == 25128545:
            print "docID:",str(docID)
            print "numOfPostings:",str(numOfPostings)
        
        for index in range(0,numOfPostings):
            byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
            (termID,staticProbability,dynamicProbability,combinedProbablity,impactScore) = unpack( "1I4f", byteString)
            if docID == 18500001 or docID == 25128545:
                print index,termID,staticProbability,dynamicProbability,combinedProbablity,impactScore
        
        num_of_bytes_read += numOfPostings * 5 * 4
        
    
    '''
    # docID: 1
    byteString = inputFileHandler.read(4 + 4)
    (docID,numOfPostings) = unpack( "2I", byteString)
    print "docID:",str(docID)
    print "numOfPostings:",str(numOfPostings)
    for index in range(0,numOfPostings):
        byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
        (termID,staticProbability,dynamicProbability,combinedProbablity,impactScore) = unpack( "1I4f", byteString)
        print index,termID,staticProbability,dynamicProbability,combinedProbablity,impactScore

    # docID: 2
    byteString = inputFileHandler.read(4 + 4)
    (docID,numOfPostings) = unpack( "2I", byteString)
    print "docID:",str(docID)
    print "numOfPostings:",str(numOfPostings)
    for index in range(0,numOfPostings):
        byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
        (termID,staticProbability,dynamicProbability,combinedProbablity,impactScore) = unpack( "1I4f", byteString)
        print index,termID,staticProbability,dynamicProbability,combinedProbablity,impactScore
    '''
    
    inputFileHandler.close()
    print "Overall:"
    print "inputFileName:",inputFileName

# been tested years ago
simpleLook()
# read1()
# read2()
# read3()
# write()

# Updated by Wei on 2014/03/11 night
# readAndWrite()
# readAndVerify()

print "End"

